{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "python36",
      "language": "python",
      "name": "python36"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.8"
    },
    "colab": {
      "name": "task1.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "_RE6488Ey1AD",
        "colab_type": "code",
        "outputId": "0a4a3138-0d8c-4bc3-e836-1f4fc2b30b9a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 54
        }
      },
      "source": [
        "'''\n",
        "实验目的：实现基于logistic/softmax regression的文本分类\n",
        "\n",
        "实验内容：\n",
        "1）简单文本特征表示（词频、n-gram、TF-IDF）、特征组合与特征选择\n",
        "2）模型相关：（随机）梯度下降、损失函数、学习率\n",
        "3）实际操作：训练集/验证集/测试集的划分、shuffle\n",
        "\n",
        "补充：\n",
        "1）更加复杂的句子级特征暂时没有构建\n",
        "2）可以使用纯numpy实现，比较懒所以用现成的库了\n",
        "\n",
        "参考：\n",
        "https://scikit-learn.org/stable/\n",
        "http://sklearn.apachecn.org/#/\n",
        "https://zhuanlan.zhihu.com/p/37157010\n",
        "\n",
        "'''"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'\\n实验目的：实现基于logistic/softmax regression的文本分类\\n\\n实验内容：\\n1）简单文本特征表示（词频、n-gram、TF-IDF）、特征组合与特征选择\\n2）模型相关：（随机）梯度下降、损失函数、学习率\\n3）实际操作：训练集/验证集/测试集的划分、shuffle\\n\\n补充：\\n1）更加复杂的句子级特征暂时没有构建\\n2）可以使用纯numpy实现，比较懒所以用现成的库了\\n\\n参考：\\nhttps://scikit-learn.org/stable/\\nhttp://sklearn.apachecn.org/#/\\nhttps://zhuanlan.zhihu.com/p/37157010\\n\\n'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 1
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U0OyDnOMzHGN",
        "colab_type": "code",
        "outputId": "0fd732b7-38d9-4147-f09a-36b99063e373",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 353
        }
      },
      "source": [
        "import os\n",
        "import sys\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "os.chdir('/content/drive/My Drive/Colab/nlp-beginner_solution/Task1/')\n",
        "\n",
        "#!pip install tensorflow-gpu==1.14.0\n",
        "#%tensorflow_version 1.14.0\n",
        "\n",
        "!ls\n",
        "!pwd\n",
        "!nvidia-smi"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n",
            "data  readme.md  task1.ipynb\n",
            "/content/drive/My Drive/Colab/nlp-beginner_solution/Task1\n",
            "Tue Nov 26 08:32:02 2019       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 440.33.01    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   33C    P8    27W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                       GPU Memory |\n",
            "|  GPU       PID   Type   Process name                             Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xJg0VCymy1AI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import torch\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import sklearn\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "\n",
        "from scipy.sparse import hstack\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "\n",
        "dir_all_data='data/task1_all_data.tsv'\n",
        "#dir_all_data='D:\\\\workspace\\\\nlp_beginer_solution\\\\Task1\\\\data\\\\task1_all_data.tsv'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sy68j3Gjy1AK",
        "colab_type": "text"
      },
      "source": [
        "#### 读取数据与数据处理"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "38JADzLry1AK",
        "colab_type": "code",
        "outputId": "2a1e24e9-591e-4d47-94c2-ee15e02b643b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 134
        }
      },
      "source": [
        "#读取数据 \n",
        "data_all=pd.read_csv(dir_all_data,sep='\\t')\n",
        "print(\"data_all.shape: \",data_all.shape)    #(156060, 4)\n",
        "print(\"data_all.keys: \",data_all.keys())   #['PhraseId', 'SentenceId', 'Phrase', 'Sentiment']\n",
        "print(data_all.head(2))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "data_all.shape:  (156060, 4)\n",
            "data_all.keys:  Index(['PhraseId', 'SentenceId', 'Phrase', 'Sentiment'], dtype='object')\n",
            "   PhraseId  ...  Sentiment\n",
            "0         1  ...          1\n",
            "1         2  ...          2\n",
            "\n",
            "[2 rows x 4 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AmuttKEiy1AN",
        "colab_type": "code",
        "outputId": "1745226f-5d99-4a1f-e21b-eb1ba19cb985",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#取出要处理的列\n",
        "x_all=data_all['Phrase']\n",
        "y_all=data_all['Sentiment']\n",
        "print(x_all.shape)   #(156060,)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(156060,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vP8r2aLPy1AP",
        "colab_type": "code",
        "outputId": "499846be-e3b3-4878-e5e7-b05322cd26da",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#划分验证集、测试集\n",
        "from sklearn.model_selection import train_test_split\n",
        "x_train, x_test,y_train,  y_test = train_test_split(x_all,y_all, test_size=0.2)\n",
        "x_train, x_val,y_train,y_val = train_test_split(x_train,y_train, test_size=0.25)\n",
        "print(x_train.shape, x_val.shape, x_test.shape)   #(93636,) (31212,) (31212,)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(93636,) (31212,) (31212,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "48yTXtsoy1AR",
        "colab_type": "text"
      },
      "source": [
        "####  接下来要提取几个特征：文本计数特征、word级别的TF-IDF特征、ngram级别的TF-IDF特征      \n",
        "#### Then---------->合并特征"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OL4UUi5Qy1AR",
        "colab_type": "code",
        "outputId": "ba896208-9710-45ec-d1b2-18c015f8d7ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#提取文本计数特征 -- 每个单词的数量\n",
        "#对文本的单词进行计数，包括文本的预处理, 分词以及过滤停用词\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "count_vect = CountVectorizer()  \n",
        "x_train_counts = count_vect.fit_transform(x_train)\n",
        "x_test_counts = count_vect.transform(x_test)\n",
        "print(x_train_counts.shape,x_test_counts.shape)  #(93636, 15188) (31212, 15188)  矩阵(句子-词汇）的维度，词表大小15188\n",
        "\n",
        "#在词汇表中一个单词的索引值对应的是该单词在整个训练的文集中出现的频率。\n",
        "#print(count_vect.vocabulary_.get(u'good'))    #5812     count_vect.vocabulary_是一个词典：word-id"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(93636, 15182) (31212, 15182)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3ge21YZy1AT",
        "colab_type": "code",
        "outputId": "d13397c9-d2fd-42cd-9065-d5492011fdc3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#提取TF-IDF特征 -- word级别的TF-IDF\n",
        "#将各文档中每个单词的出现次数除以该文档中所有单词的总数：这些新的特征称之为词频tf。\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "tfidf_transformer = TfidfVectorizer(analyzer='word',max_features=50000)\n",
        "tfidf_transformer.fit(x_train)\n",
        "x_train_tfidf_word = tfidf_transformer.transform(x_train)\n",
        "x_test_tfidf_word = tfidf_transformer.transform(x_test)\n",
        "print(x_train_tfidf_word.shape,x_test_tfidf_word.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(93636, 15182) (31212, 15182)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ex5SD51Oy1AV",
        "colab_type": "code",
        "outputId": "94c55b2c-7406-42a2-911a-13353211e4ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#提取TF-IDF特征 - ngram级别的TF-IDF\n",
        "#将各文档中每个单词的出现次数除以该文档中所有单词的总数：这些新的特征称之为词频tf。\n",
        "from sklearn.feature_extraction.text import TfidfTransformer\n",
        "tfidf_transformer = TfidfVectorizer(analyzer='word',ngram_range=(2,3),max_features=50000)\n",
        "tfidf_transformer.fit(x_train)\n",
        "x_train_tfidf_ngram = tfidf_transformer.transform(x_train)\n",
        "x_test_tfidf_ngram = tfidf_transformer.transform(x_test)\n",
        "print(x_train_tfidf_ngram.shape, x_test_tfidf_ngram.shape)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(93636, 50000) (31212, 50000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SYc3rsS0y1AW",
        "colab_type": "code",
        "outputId": "00b7d756-5288-427e-ed54-1b86ddff8bfc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#合并特征（特征组合与特征选择）\n",
        "#from scipy.sparse import hstack\n",
        "train_features=x_train_counts\n",
        "test_features=x_test_counts\n",
        "\n",
        "train_features = hstack([x_train_counts,x_train_tfidf_word, x_train_tfidf_ngram])\n",
        "test_features = hstack([x_test_counts,x_test_tfidf_word ,x_test_tfidf_ngram])\n",
        "\n",
        "print(train_features.shape)   #特征的最终维度\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(93636, 80364)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vnOT572zy1AY",
        "colab_type": "text"
      },
      "source": [
        "#### 模型构建与训练"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1xf13hXZy1AZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#训练分类器\n",
        "\n",
        "#逻辑回归\n",
        "# from sklearn.linear_model import LogisticRegression\n",
        "# clf = LogisticRegression(random_state=0, \n",
        "#                          solver='sag', #优化算法：liblinear、lbfgs、newton-cg、sag\n",
        "#                          multi_class='multinomial' #分类方式：multinomial、ovr\n",
        "# )\n",
        "\n",
        "#朴素贝叶斯\n",
        "#from sklearn.naive_bayes import MultinomialNB\n",
        "#clf = MultinomialNB().fit(train_features, y_train)\n",
        "\n",
        "\n",
        "#SGDClassifier是一系列采用了梯度下降来求解参数的算法的集合，默认是SVM\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "clf = SGDClassifier(alpha=0.001,\n",
        "                    loss='log',    #hinge代表SVM，log是逻辑回归\n",
        "                    early_stopping=True,\n",
        "                    eta0=0.001,\n",
        "                    learning_rate='adaptive', #constant、optimal、invscaling、adaptive\n",
        "                    max_iter=100 \n",
        "                   )"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L1hZ8wcJy1Ab",
        "colab_type": "code",
        "outputId": "e97e56b4-34ad-4bba-fbad-77b1b0d1852d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 118
        }
      },
      "source": [
        "#打乱数据，训练\n",
        "from sklearn.utils import shuffle\n",
        "train_features,y_train=shuffle(train_features,y_train )\n",
        "\n",
        "clf.fit(train_features, y_train)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SGDClassifier(alpha=0.001, average=False, class_weight=None,\n",
              "              early_stopping=True, epsilon=0.1, eta0=0.001, fit_intercept=True,\n",
              "              l1_ratio=0.15, learning_rate='adaptive', loss='log', max_iter=100,\n",
              "              n_iter_no_change=5, n_jobs=None, penalty='l2', power_t=0.5,\n",
              "              random_state=None, shuffle=True, tol=0.001,\n",
              "              validation_fraction=0.1, verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FRj3BCrBy1Ad",
        "colab_type": "text"
      },
      "source": [
        "#### 测试过程"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3iRnzHHzy1Ae",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#测试过程\n",
        "predict = clf.predict(test_features)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VzNyfyySy1Af",
        "colab_type": "code",
        "outputId": "eaa245e9-8c9f-49a1-8f58-99261cffebf4",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#测试集的评估\n",
        "print(np.mean(predict == y_test))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.5432205561963347\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j2n3obHpy1Ah",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jfhsctMty1Aj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}